---
title: "Replication File for 'Experiments Offering Social Media Users the Algorithmic Choice to Avoid Toxic Political Content'"
output: 
    pdf_document: 
      keep_tex:  false
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(
  echo = FALSE, 
  warning = FALSE,
  error = FALSE, 
  message = FALSE,
  cache = FALSE,
  fig.align = 'center'
  )
```

```{r}
## Load packages
library(kableExtra)
library(sjPlot)
library(sjmisc)
library(sjlabelled)
library(tidyverse)
library(stargazer)
library(scales)
library(reshape2)  
library(patchwork)
library(nnet)

## Load data 1 and 2:

data <- read_csv("data_for_replication.csv") 
data2 <- read_csv("data2_for_replication.csv") 

## Create the data to use in models
## Prepping data for opt-ins.
treated_data <- data %>% filter(treatment_condition == 1)
treated_yes <- treated_data %>% filter(algo_selected == 1)
treated_no <- treated_data %>% filter(algo_selected == 0)
control_data <- data %>% filter(treatment_condition == 0)

rhs_prop_formula <- "SM_shadow_banning_problem_numeric + 
                    satisfied_with_sm + 
                    online_bullying_problem_numeric + 
                    college + 
                    frequency_of_sm_use_numeric + 
                    dem_and_leaning + 
                    SM_banning_users_problem_numeric + 
                    white_only + 
                    misinfo_problem_numeric + 
                    female + 
                    strong_party_id2 + 
                    tone_civility_problem_numeric + 
                    age +
                    talk_to_others_abt_politics_frequently +
                    talk_to_others_abt_politics_infrequently"

rhs_reg_formula <- "treatment_condition + 
                    dem_and_leaning +
                    college +
                    strong_party_id2 +
                    age +
                    female +
                    white_only +
                    frequency_of_sm_use_numeric +
                    satisfied_with_sm +
                    misinfo_problem_numeric +
                    SM_banning_users_problem_numeric +
                    SM_shadow_banning_problem_numeric +
                    online_bullying_problem_numeric +
                    tone_civility_problem_numeric +
                    talk_to_others_abt_politics_frequently +
                    talk_to_others_abt_politics_infrequently"

algo_selected_model <- glm(reformulate(rhs_prop_formula, 
                                       "algo_selected"), 
                           data = treated_data,
                           family = "binomial")

control_scores <- predict(algo_selected_model, newdata = control_data, type = "response")

# adding prop scores to control
control_data <- control_data %>% mutate(
  algo_selected_att = control_scores/(1-control_scores),
  algo_selected_prT = control_scores,
  algo_selected_thr = control_scores>.5,
)

treatment_yes_and_control <- bind_rows(treated_yes,control_data) #GT edit
treatment_yes_and_control <- treatment_yes_and_control %>% 
  mutate(across(starts_with("algo_selected_"),function(x) ifelse(is.na(x),1,x)))


algo_NOT_selected_model <- glm(reformulate(rhs_prop_formula, 
                                           "algo_NOT_selected"), 
                               data = treated_data,
                               family = "binomial")

control_scores <- predict(algo_NOT_selected_model, newdata = control_data, type = "response")

# adding prop scores to control
control_data <- control_data %>% mutate(
  algo_not_selected_att = control_scores/(1-control_scores),
  algo_not_selected_prT = control_scores,
  algo_not_selected_thr = control_scores>.5,
)

treatment_no_and_control <- bind_rows(treated_no,control_data) #GT edit
treatment_no_and_control <- treatment_no_and_control %>% 
  mutate(across(starts_with("algo_not_selected_"),function(x) ifelse(is.na(x),1,x)))

## Data 2 code
# For follow-up study
## Prepping data2 for opt-ins.
treated_data2 <- data2 %>% filter(treatment_condition == 1)
treated_yes <- treated_data2 %>% filter(algo_selected == 1)
treated_no <- treated_data2 %>% filter(algo_selected == 0)
control_data2 <- data2 %>% filter(treatment_condition == 0)

algo_selected_model2 <- glm(reformulate(rhs_prop_formula, 
                                        "algo_selected"), 
                            data = treated_data2,
                            family = "binomial")

control_scores2 <- predict(algo_selected_model2, newdata = control_data2, type = "response")

# adding prop scores to control
control_data2 <- control_data2 %>% mutate(
  algo_selected_att = control_scores2/(1-control_scores2),
  algo_selected_prT = control_scores2,
  algo_selected_thr = control_scores2>.5,
)

treatment_yes_and_control2 <- bind_rows(treated_yes,control_data2) #GT edit
treatment_yes_and_control2 <- treatment_yes_and_control2 %>% 
  mutate(across(starts_with("algo_selected_"),function(x) ifelse(is.na(x),1,x)))


algo_NOT_selected_model2 <- glm(reformulate(rhs_prop_formula, 
                                            "algo_NOT_selected"), 
                                data = treated_data2,
                                family = "binomial")

control_scores2 <- predict(algo_NOT_selected_model2, newdata = control_data2, type = "response")

# adding prop scores to control
control_data2 <- control_data2 %>% mutate(
  algo_not_selected_att = control_scores2/(1-control_scores2),
  algo_not_selected_prT = control_scores2,
  algo_not_selected_thr = control_scores2>.5,
)

treatment_no_and_control2 <- bind_rows(treated_no,control_data2) #GT edit
treatment_no_and_control2 <- treatment_no_and_control2 %>% 
  mutate(across(starts_with("algo_not_selected_"),function(x) ifelse(is.na(x),1,x)))
```

# Results

## Overall effect of choice.

\clearpage 

```{r}

m1_a_1 <- {lm(
  reformulate(rhs_reg_formula, 
              "satisfied_with_platform"), 
  data = data, 
)}
m1_a_2 <- {lm(
  reformulate(rhs_reg_formula, 
              "likely_to_use_platform"), 
  data = data, 
)}
m1_a_3 <- {lm(
  reformulate(rhs_reg_formula, 
              "would_recommend_platform_to_friend"), 
  data = data, 
)}

m1_b_1 <- {lm(
  reformulate(rhs_reg_formula, 
              "avg_hostility_score_all_posts"), 
  data = data
)}
m1_b_2 <- {lm(
  reformulate(rhs_reg_formula, 
              "political_tweets_hostile_avg"), 
  data = data
)}
m1_b_3 <- {lm(
  reformulate(rhs_reg_formula, 
              "nonpolitical_avg_hostile"), 
  data = data
)}
m1_b_4 <- {lm(
  reformulate(rhs_reg_formula, 
              "in_party_avg_hostile"), 
  data = data
)}
m1_b_5 <- {lm(
  reformulate(rhs_reg_formula, 
              "out_party_avg_hostile"), 
  data = data
)}
```

```{r overall-effects-of-choice, fig.cap = "Figure 4 of main article", results = 'hold', out.extra='angle=90', out.width = '100%', out.height='100%', fig.width = 9, fig.height = 4}

model_summary_for_plot <- function(...) {
  models_list <- list(...)
  # Apply the function to each model in the list and row-bind the results
  output_table <- purrr::map_dfr(models_list, function(model) {
    model %>% 
      summary() %>% 
      .$coefficients %>%
      as_tibble() %>%
      mutate(variable = row.names(summary(model)$coefficients)) %>%
      rename(
        coefficient = Estimate,
        std_error = `Std. Error`,
        t_value = `t value`,
        p_value = `Pr(>|t|)`
      ) %>%
      relocate(variable, .before = 1) %>%
      mutate(
        lower_95 = coefficient - qt(0.975, model$df.residual)*std_error,
        upper_95 = coefficient + qt(0.975, model$df.residual)*std_error,
        lower_90 = coefficient - qt(0.95, model$df.residual)*std_error,
        upper_90 = coefficient + qt(0.95, model$df.residual)*std_error,
        model = names(model$model)[[1]]
      )  %>%
      pivot_longer(cols = c(lower_90, upper_90, lower_95, upper_95, coefficient), names_to = "x_axis", values_to = "value") %>% 
      filter(variable == "treatment_condition") %>%
      separate(x_axis, into = c("limit_type", "conf_level"), sep = "_", fill = "right") %>%
      mutate(conf_level = ifelse(is.na(conf_level), "coefficient", conf_level))
  })
}

m1_a_plot_data <- model_summary_for_plot(m1_a_1, m1_a_2, m1_a_3) %>% 
  mutate(facet = "A: Effect of choice on evaluation of platform") %>% 
  mutate(model = factor(model, levels = c(  "likely_to_use_platform",
                                            "would_recommend_platform_to_friend",
                                            "satisfied_with_platform"),
                        labels = c(
                                "likely_to_use_platform" = "Using the platform",
                                "would_recommend_platform_to_friend" = "Recommending to friend",
                                "satisfied_with_platform" = "Satisfaction with\nConversationCircle"
                                )
                        ))

m1_a_plot <- ggplot(m1_a_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m1_a_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m1_a_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust=0)) +
  labs(x = "", y = "") 

m1_b_plot_data <- model_summary_for_plot(m1_b_1, m1_b_2, m1_b_3, m1_b_4, m1_b_5) %>% 
  mutate(facet = "B: Effect of choice on hostility ratings of content") %>% 
  mutate(model = factor(model, levels = c(  "out_party_avg_hostile",
                                            "in_party_avg_hostile",
                                            "nonpolitical_avg_hostile",
                                            "political_tweets_hostile_avg",
                                            "avg_hostility_score_all_posts"), 
                        labels=c(
    "out_party_avg_hostile" = "Out-party political posts",
    "in_party_avg_hostile" = "In-party political posts",
    "nonpolitical_avg_hostile" = "Non-partisan posts",
    "political_tweets_hostile_avg" = "Political posts",
    "avg_hostility_score_all_posts" = "All posts combined"
  ))) 

m1_b_plot <- ggplot(m1_b_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m1_b_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m1_b_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none",  axis.text.y = element_text(hjust=0)) +
  labs(x = "Estimate", y = "") #+
  # scale_y_discrete(labels=c(
  #   "out_party_avg_hostile" = "Out-party political\nposts",
  #   "in_party_avg_hostile" = "In-party political\nposts",
  #   "nonpolitical_avg_hostile" = "Non-partisan posts",
  #   "political_tweets_hostile_avg" = "Political posts",
  #   "avg_hostility_score_all_posts" = "All posts combined"
  # ))

# # Compute the common x range for both plots
# xmin <- min(min(m1_a_plot_data$value, na.rm = TRUE), min(m1_b_plot_data$value, na.rm = TRUE))
# xmax <- max(max(m1_a_plot_data$value, na.rm = TRUE), max(m1_b_plot_data$value, na.rm = TRUE))
# 
# {m1_a_plot + coord_cartesian(xlim = c(xmin, xmax)) + 
#     ggtitle("A: Effect of choice on evaluation of platform")} / 
#   {m1_b_plot + coord_cartesian(xlim = c(xmin, xmax)) + 
#       ggtitle("B: Effect of choice on hostility ratings of content")} & theme(panel.border = element_rect(colour = "black", fill=NA))

m1_plot_data <- rbind(m1_a_plot_data, m1_b_plot_data)

ggplot(m1_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m1_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m1_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  labs(x = "Estimate", y = "") +
  facet_wrap(~facet, nrow = 2, scales = "free_y") +
    theme(legend.position = "none",  
        strip.text.x = element_text(face= "bold", size = 12)
        ) 
  
  
```

### Regression tables for overall effects of choice

```{r,include = FALSE}
# ============================================================
# STUDY 1: OVERALL EFFECT OF CHOICE
# ============================================================

# Platform Evaluation (Study 1 Overall)
stargazer::stargazer(m1_a_1, m1_a_2, m1_a_3,
  type = 'text',
  title = "Study 1: Overall Effect of Choice on Platform Evaluation",
  column.labels = c('Satisfaction', 'Likely to Use', 'Recommend'),
  dep.var.labels = 'Platform Evaluation',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"#,
  #omit.stat = c("f", "ser")
)

# Hostility Ratings (Study 1 Overall)
stargazer::stargazer(m1_b_1, m1_b_2, m1_b_3, m1_b_4, m1_b_5,
  type = 'text',
  title = "Study 1: Overall Effect of Choice on Hostility Ratings",
  column.labels = c('All Posts', 'Political', 'Non-partisan', 'In-party', 'Out-party'),
  dep.var.labels = 'Mean Hostility Rating',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"#,
  #omit.stat = c("f", "ser")
)

```


## Proportion of participants who opt-in

```{r percent-opt-ins, fig.cap = "Figure 3 of main article", results = 'hold', echo = F, out.height='80%', out.width='80%', fig.align = 'center'}
treated_data %>% 
  filter(!is.na(treated_choice_made)) %>% 
  ggplot(aes(x = as.factor(algo_selected), y = 100*after_stat(count)/nrow(.), 
             fill = as.factor(algo_selected)
             )) +
  geom_bar() + 
  labs(x = "", y = "% of Respondents Given Choice\nto Avoid Toxic Content", fill = "") +
  theme_classic() +
  scale_fill_discrete(labels = c("Opted out", "Opted in")) +
  #scale_fill_manual(values=c("#811c1c", "#274e13")) +
  scale_x_discrete(labels = c("Opted out", "Opted in")) +
  theme(axis.text.x=element_text(size=12), 
        axis.text.y=element_text(size=12),
        axis.title = element_text(size = 15),
        legend.position="none") 
``` 

\clearpage 

\clearpage

## Coeff plot for Table SI 8 in supplement and Figure 5 in manuscript

\clearpage 

```{r coeff-plot-algoselected, results = 'hold', echo = F, out.width = '100%', out.height='100%', fig.width = 10, fig.height = 5, fig.cap = "algo selected coeff plot. USE THIS PLOT", out.extra='angle=90'}
jtools::plot_coefs(
  algo_selected_model,
inner_ci_level=0.9, 
  scale = TRUE, 
  robust = TRUE,
exp = T,
   coefs = c(
  "Social media has a shadow banning broblem" = "SM_shadow_banning_problem_numeric",
  "Satisfied with social media" = "satisfied_with_sm",
  "Social media has an online bullying problem" = "online_bullying_problem_numeric",
  "Frequency of social media Use" = "frequency_of_sm_use_numeric",
  "College educated" = "college",
  "Democrat" = "dem_and_leaning",
  "Social media has a banning users problem" = "SM_banning_users_problem_numeric",
  "White" = "white_only",
  "Social media has a misinformation problem" = "misinfo_problem_numeric",
  "Female" = "female",
  "Strong party identification" = "strong_party_id2",
  "Age" = "age",
  "Social media has a tone and civility Problem" = "tone_civility_problem_numeric",
  "Frequently discusses politics with others" = "talk_to_others_abt_politics_frequently",
  "Infrequently discusses politics with others" = "talk_to_others_abt_politics_infrequently"
),
  legend.title = "",
  # model.names = c(
  #   
  # ),
  point.shape = FALSE,
  colors = "#020079"
) + 
  xlab("Odds ratio of opting-in to viewing less toxic content") +
  theme(legend.position="None") +
  theme(axis.text.y = element_text(hjust = 1)) 
  
  #guides(color=guide_legend(nrow=2,byrow=TRUE)) 
```

```{r, include = FALSE}
# ============================================================
# PROPENSITY MODEL (Logistic Regression for Opt-In)
# ============================================================

stargazer::stargazer(algo_selected_model,
  type = 'text',
  title = "Logistic Regression: Predictors of Opting-In to Toxicity Filter",
  dep.var.labels = 'Opted In (1 = Yes)',
  covariate.labels = c(
    "Shadow-banning problem",
    "Satisfied with SM",
    "Online bullying problem",
    "College educated",
    "Frequency of SM use",
    "Democrat",
    "Banning users problem",
    "White",
    "Misinformation problem",
    "Female",
    "Strong party ID",
    "Tone/civility problem",
    "Age",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"#,
  #omit.stat = c("f", "ser")
)
```




\clearpage 

## Effect of choice among opt-ins only

\clearpage 

```{r effects-of-choice-optins-only, results = 'hold', echo = F, out.extra='angle=90', out.width = '100%', out.height='100%', fig.width = 9, fig.height = 4, fig.cap = "Figure 6 of main article"}
m2_a_1 <-  {lm(
    reformulate(rhs_reg_formula, 
                "satisfied_with_platform"), 
  data = treatment_yes_and_control, 
weights = algo_selected_prT
)}
m2_a_2 <-  {lm(
    reformulate(rhs_reg_formula, 
                "likely_to_use_platform"), 
  data = treatment_yes_and_control, 
weights = algo_selected_prT
)}
m2_a_3 <-  {lm(
    reformulate(rhs_reg_formula, 
                "would_recommend_platform_to_friend"), 
  data = treatment_yes_and_control, 
weights = algo_selected_prT
)}

m2_b_1 <- {lm(
    reformulate(rhs_reg_formula, 
                "avg_hostility_score_all_posts"), 
    data = treatment_yes_and_control, 
weights = algo_selected_prT
  )}
m2_b_2 <- {lm(
    reformulate(rhs_reg_formula, 
                "political_tweets_hostile_avg"), 
    data = treatment_yes_and_control, 
weights = algo_selected_prT
  )}
m2_b_3 <- {lm(
    reformulate(rhs_reg_formula, 
                "nonpolitical_avg_hostile"), 
    data = treatment_yes_and_control, 
weights = algo_selected_prT
  )}
m2_b_4 <- {lm(
    reformulate(rhs_reg_formula, 
                "in_party_avg_hostile"), 
    data = treatment_yes_and_control, 
weights = algo_selected_prT
  )}
m2_b_5 <- {lm(
    reformulate(rhs_reg_formula, 
                "out_party_avg_hostile"), 
    data = treatment_yes_and_control, 
weights = algo_selected_prT
  )}

m2_a_plot_data <- model_summary_for_plot(m2_a_1, m2_a_2, m2_a_3) %>% 
  mutate(facet = "A: Effect of choice on evaluation of platform\namong those who chose to view less toxic political posts and\n those who likely would have chosen had they been offered the choice") %>% 
  mutate(model = factor(model, levels = c(  "likely_to_use_platform",
                                            "would_recommend_platform_to_friend",
                                            "satisfied_with_platform"),
                        labels = c(
                                "likely_to_use_platform" = "Using the platform",
                                "would_recommend_platform_to_friend" = "Recommending to friend",
                                "satisfied_with_platform" = "Satisfaction with\nConversationCircle"
                                )
                        ))

m2_a_plot <- ggplot(m2_a_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m2_a_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m2_a_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust=0)) +
  labs(x = "", y = "") 

m2_b_plot_data <- model_summary_for_plot(m2_b_1, m2_b_2, m2_b_3, m2_b_4, m2_b_5) %>% 
  mutate(facet = "B: Effect of choice on hostility ratings of content\namong those who chose to view less toxic political posts and\n those who likely would have chosen had they been offered the choice") %>% 
  mutate(model = factor(model, levels = c(  "out_party_avg_hostile",
                                            "in_party_avg_hostile",
                                            "nonpolitical_avg_hostile",
                                            "political_tweets_hostile_avg",
                                            "avg_hostility_score_all_posts"), 
                        labels=c(
    "out_party_avg_hostile" = "Out-party political posts",
    "in_party_avg_hostile" = "In-party political posts",
    "nonpolitical_avg_hostile" = "Non-partisan posts",
    "political_tweets_hostile_avg" = "Political posts",
    "avg_hostility_score_all_posts" = "All posts combined"
  ))) 

m2_b_plot <- ggplot(m2_b_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m2_b_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m2_b_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none",  axis.text.y = element_text(hjust=0)) +
  labs(x = "Estimate", y = "") 

m2_plot_data <- rbind(m2_a_plot_data, m2_b_plot_data)

ggplot(m2_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m2_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m2_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  labs(x = "Estimate", y = "") +
  facet_wrap(~facet, nrow = 2, scales = "free_y") +
    theme(legend.position = "none",  
        strip.text.x = element_text(face= "bold", size = 12)
        ) 

```

```{r, include = FALSE}
# ============================================================
# STUDY 1: EFFECT AMONG OPT-INS (Propensity Weighted)
# ============================================================

# Platform Evaluation (Study 1 Opt-ins)
stargazer::stargazer(m2_a_1, m2_a_2, m2_a_3,
  type = 'text',
  title = "Study 1: Effect of Choice Among Opt-Ins on Platform Evaluation",
  column.labels = c('Satisfaction', 'Likely to Use', 'Recommend'),
  dep.var.labels = 'Platform Evaluation (Propensity Weighted)',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"#,
  #omit.stat = c("f", "ser")
)

# Hostility Ratings (Study 1 Opt-ins)
stargazer::stargazer(m2_b_1, m2_b_2, m2_b_3, m2_b_4, m2_b_5,
  type = 'text',
  title = "Study 1: Effect of Choice Among Opt-Ins on Hostility Ratings",
  column.labels = c('All Posts', 'Political', 'Non-political', 'In-party', 'Out-party'),
  dep.var.labels = 'Mean Hostility Rating (Propensity Weighted)',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"#,
  #omit.stat = c("f", "ser")
)
```


\clearpage 

# Study 2: Overall effect of choice

\clearpage

```{r s2-overall-effects-of-choice, fig.cap = "Study 2: Overall effects of choice. This includes both opt-ins and opt-outs. We don't report this in main manuscript. But include it in Supplemental Information 2 Follow-up Study", results = 'hold', out.extra='angle=90', out.width = '100%', out.height='100%', fig.width = 9, fig.height = 4}
s2_m1_a_1 <-  {lm(
    reformulate(rhs_reg_formula, 
                "satisfied_with_platform"), 
  data = data2, 
)}
s2_m1_a_2 <-  {lm(
    reformulate(rhs_reg_formula, 
                "likely_to_use_platform"), 
  data = data2, 
)}

s2_m1_a_3 <-  {lm(
    reformulate(rhs_reg_formula, 
                "would_recommend_platform_to_friend"), 
  data = data2, 
)}
  
s2_m1_b_1 <- {lm(
    reformulate(rhs_reg_formula, 
                "avg_hostility_score_all_posts"), 
    data = data2
  )}
s2_m1_b_2 <- {lm(
    reformulate(rhs_reg_formula, 
                "political_tweets_hostile_avg"), 
    data = data2
  )}
s2_m1_b_3 <- {lm(
    reformulate(rhs_reg_formula, 
                "nonpolitical_avg_hostile"), 
    data = data2
  )}
s2_m1_b_4 <- {lm(
    reformulate(rhs_reg_formula, 
                "in_party_avg_hostile"), 
    data = data2
  )}
s2_m1_b_5 <- {lm(
    reformulate(rhs_reg_formula, 
                "out_party_avg_hostile"), 
    data = data2
  )}

s2_m1_a_plot_data <- model_summary_for_plot(s2_m1_a_1, s2_m1_a_2, s2_m1_a_3) %>% 
  mutate(facet = "A: Effect of choice on evaluation of platform") %>% 
  mutate(model = factor(model, levels = c(  "likely_to_use_platform",
                                            "would_recommend_platform_to_friend",
                                            "satisfied_with_platform"),
                        labels = c(
                                "likely_to_use_platform" = "Using the platform",
                                "would_recommend_platform_to_friend" = "Recommending to friend",
                                "satisfied_with_platform" = "Satisfaction with\nConversationCircle"
                                )
                        ))

s2_m1_a_plot <- ggplot(s2_m1_a_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m1_a_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m1_a_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust=0)) +
  labs(x = "", y = "") 

s2_m1_b_plot_data <- model_summary_for_plot(s2_m1_b_1, s2_m1_b_2, s2_m1_b_3, s2_m1_b_4, s2_m1_b_5) %>% 
  mutate(facet = "B: Effect of choice on hostility ratings of content") %>% 
  mutate(model = factor(model, levels = c(  "out_party_avg_hostile",
                                            "in_party_avg_hostile",
                                            "nonpolitical_avg_hostile",
                                            "political_tweets_hostile_avg",
                                            "avg_hostility_score_all_posts"), 
                        labels=c(
    "out_party_avg_hostile" = "Out-party political posts",
    "in_party_avg_hostile" = "In-party political posts",
    "nonpolitical_avg_hostile" = "Non-partisan posts",
    "political_tweets_hostile_avg" = "Political posts",
    "avg_hostility_score_all_posts" = "All posts combined"
  ))) 

s2_m1_b_plot <- ggplot(s2_m1_b_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m1_b_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m1_b_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none",  axis.text.y = element_text(hjust=0)) +
  labs(x = "Estimate", y = "") 

s2_m1_plot_data <- rbind(s2_m1_a_plot_data, s2_m1_b_plot_data)

ggplot(s2_m1_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m1_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m1_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  labs(x = "Estimate", y = "") +
  facet_wrap(~facet, nrow = 2, scales = "free_y") +
    theme(legend.position = "none",  
        strip.text.x = element_text(face= "bold", size = 12)
        ) 
```

```{r, include = FALSE}
# ============================================================
# STUDY 2: OVERALL EFFECT OF CHOICE
# ============================================================

# Platform Evaluation (Study 2 Overall)
stargazer::stargazer(s2_m1_a_1, s2_m1_a_2, s2_m1_a_3,
  type = 'text',
  title = "Study 2: Overall Effect of Choice on Platform Evaluation",
  column.labels = c('Satisfaction', 'Likely to Use', 'Recommend'),
  dep.var.labels = 'Platform Evaluation',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"
)

# Hostility Ratings (Study 2 Overall)
stargazer::stargazer(s2_m1_b_1, s2_m1_b_2, s2_m1_b_3, s2_m1_b_4, s2_m1_b_5,
  type = 'text',
  title = "Study 2: Overall Effect of Choice on Hostility Ratings",
  column.labels = c('All Posts', 'Political', 'Non-political', 'In-party', 'Out-party'),
  dep.var.labels = 'Mean Hostility Rating',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"
)
```


\clearpage

## Proportion of participants who opt-in from study 2

```{r s2-percent-opt-ins, fig.cap = "Figure 10 of main article (in Appendix A)", results = 'hold', echo = F, out.height='80%', out.width='80%', fig.align = 'center'}
treated_data2 %>% 
  filter(!is.na(treated_choice_made)) %>% 
  ggplot(aes(x = as.factor(algo_selected), y = 100*after_stat(count)/nrow(.), 
             fill = as.factor(algo_selected)
             )) +
  geom_bar() + 
  labs(x = "", y = "% of Respondents Given Choice\nto Avoid Toxic Content", fill = "") +
  theme_classic() +
  scale_fill_discrete(labels = c("Opted out", "Opted in")) +
  #scale_fill_manual(values=c("#811c1c", "#274e13")) +
  scale_x_discrete(labels = c("Opted out", "Opted in")) +
  theme(axis.text.x=element_text(size=12), 
        axis.text.y=element_text(size=12),
        axis.title = element_text(size = 15),
        legend.position="none") 
``` 

```{r, include = FALSE}
stargazer::stargazer(algo_selected_model2,
  type = 'text',
  title = "Logistic Regression: Predictors of Opting-In to Toxicity Filter in Follow up study",
  dep.var.labels = 'Opted In (1 = Yes)',
  covariate.labels = c(
    "Shadow-banning problem",
    "Satisfied with SM",
    "Online bullying problem",
    "College educated",
    "Frequency of SM use",
    "Democrat",
    "Banning users problem",
    "White",
    "Misinformation problem",
    "Female",
    "Strong party ID",
    "Tone/civility problem",
    "Age",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize",
  omit.stat = c("f", "ser")
)
```


\clearpage 


## Study 2: Effect of choice among opt-ins only

\clearpage

```{r s2-effects-of-choice-optins-only, results = 'hold', echo = F, out.extra='angle=90', out.width = '100%', out.height='100%', fig.width = 9, fig.height = 4, fig.cap = "Study 2 Effects of choice on opt-ins."}
s2_m2_a_1 <- {lm(
    reformulate(rhs_reg_formula, 
                "satisfied_with_platform"), 
  data = treatment_yes_and_control2, 
weights = algo_selected_prT 
)}
s2_m2_a_2 <- {lm(
    reformulate(rhs_reg_formula, 
                "likely_to_use_platform"), 
  data = treatment_yes_and_control2, 
weights = algo_selected_prT
)}
s2_m2_a_3 <- {lm(
    reformulate(rhs_reg_formula, 
                "would_recommend_platform_to_friend"), 
  data = treatment_yes_and_control2, 
weights = algo_selected_prT
)}
  
s2_m2_b_1 <- {lm(
    reformulate(rhs_reg_formula, 
                "avg_hostility_score_all_posts"), 
    data = treatment_yes_and_control2, 
weights = algo_selected_prT
  )}
s2_m2_b_2 <- {lm(
    reformulate(rhs_reg_formula, 
                "political_tweets_hostile_avg"), 
    data = treatment_yes_and_control2, 
weights = algo_selected_prT
  )}
s2_m2_b_3 <- {lm(
    reformulate(rhs_reg_formula, 
                "nonpolitical_avg_hostile"), 
    data = treatment_yes_and_control2, 
weights = algo_selected_prT
  )}
s2_m2_b_4 <- {lm(
    reformulate(rhs_reg_formula, 
                "in_party_avg_hostile"), 
    data = treatment_yes_and_control2, 
weights = algo_selected_prT
  )}
s2_m2_b_5 <- {lm(
    reformulate(rhs_reg_formula, 
                "out_party_avg_hostile"), 
    data = treatment_yes_and_control2, 
weights = algo_selected_prT
)}

s2_m2_a_plot_data <- model_summary_for_plot(s2_m2_a_1, s2_m2_a_2, s2_m2_a_3) %>% 
  mutate(facet = "A: Effect of choice on evaluation of platform in our follow-up study,\namong those who chose to view less toxic political posts and\n those who likely would have chosen had they been offered the choice") %>% 
  mutate(model = factor(model, levels = c(  "likely_to_use_platform",
                                            "would_recommend_platform_to_friend",
                                            "satisfied_with_platform"),
                        labels = c(
                                "likely_to_use_platform" = "Using the platform",
                                "would_recommend_platform_to_friend" = "Recommending to friend",
                                "satisfied_with_platform" = "Satisfaction with\nConversationCircle"
                                )
                        ))

s2_m2_a_plot <- ggplot(s2_m2_a_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m2_a_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m2_a_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust=0)) +
  labs(x = "", y = "") 

s2_m2_b_plot_data <- model_summary_for_plot(s2_m2_b_1, s2_m2_b_2, s2_m2_b_3, s2_m2_b_4, s2_m2_b_5) %>% 
  mutate(facet = "B: Effect of choice on hostility ratings of content in our follow-up study,\namong those who chose to view less toxic political posts and\n those who likely would have chosen had they been offered the choice") %>% 
  mutate(model = factor(model, levels = c(  "out_party_avg_hostile",
                                            "in_party_avg_hostile",
                                            "nonpolitical_avg_hostile",
                                            "political_tweets_hostile_avg",
                                            "avg_hostility_score_all_posts"), 
                        labels=c(
    "out_party_avg_hostile" = "Out-party political posts",
    "in_party_avg_hostile" = "In-party political posts",
    "nonpolitical_avg_hostile" = "Non-partisan posts",
    "political_tweets_hostile_avg" = "Political posts",
    "avg_hostility_score_all_posts" = "All posts combined"
  ))) 

s2_m2_b_plot <- ggplot(s2_m2_b_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m2_b_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m2_b_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none",  axis.text.y = element_text(hjust=0)) +
  labs(x = "Estimate", y = "") 

s2_m2_plot_data <- rbind(s2_m2_a_plot_data, s2_m2_b_plot_data)

ggplot(s2_m2_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m2_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m2_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  labs(x = "Estimate", y = "") +
  facet_wrap(~facet, nrow = 2, scales = "free_y") +
    theme(legend.position = "none",  
        strip.text.x = element_text(face= "bold", size = 12)
        ) 
```

```{r, include = FALSE}
# Platform Evaluation (Study 2 Opt-ins)
stargazer::stargazer(s2_m2_a_1, s2_m2_a_2, s2_m2_a_3,
  type = 'text',
  title = "Study 2: Effect of Choice Among Opt-Ins on Platform Evaluation",
  column.labels = c('Satisfaction', 'Likely to Use', 'Recommend'),
  dep.var.labels = 'Platform Evaluation (Propensity Weighted)',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE
)

# Hostility Ratings (Study 2 Opt-ins)
stargazer::stargazer(s2_m2_b_1, s2_m2_b_2, s2_m2_b_3, s2_m2_b_4, s2_m2_b_5,
  type = 'text',
  title = "Study 2: Effect of Choice Among Opt-Ins on Hostility Ratings",
  column.labels = c('All Posts', 'Political', 'Non-political', 'In-party', 'Out-party'),
  dep.var.labels = 'Mean Hostility Rating (Propensity Weighted)',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE
)

```


\clearpage

# Effect of choice on those who would _opt-out_

## Study 1

```{r effects-of-choice-optouts-only, results = 'hold', echo = F, out.extra='angle=90', out.width = '100%', out.height='100%', fig.width = 9, fig.height = 4, fig.cap = "Figure X of supplemental information Y"}
m3_a_1 <-  {lm(
    reformulate(rhs_reg_formula, 
                "satisfied_with_platform"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
)}
m3_a_2 <-  {lm(
    reformulate(rhs_reg_formula, 
                "likely_to_use_platform"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
)}
m3_a_3 <-  {lm(
    reformulate(rhs_reg_formula, 
                "would_recommend_platform_to_friend"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
)}

m3_b_1 <- {lm(
    reformulate(rhs_reg_formula, 
                "avg_hostility_score_all_posts"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
  )}
m3_b_2 <- {lm(
    reformulate(rhs_reg_formula, 
                "political_tweets_hostile_avg"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
)}

m3_b_3 <- {lm(
    reformulate(rhs_reg_formula, 
                "nonpolitical_avg_hostile"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
)}

m3_b_4 <- {lm(
    reformulate(rhs_reg_formula, 
                "in_party_avg_hostile"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
)}

m3_b_5 <- {lm(
    reformulate(rhs_reg_formula, 
                "out_party_avg_hostile"), 
  data = treatment_no_and_control, 
weights = algo_not_selected_prT
  )}

m3_a_plot_data <- model_summary_for_plot(m3_a_1, m3_a_2, m3_a_3) %>% 
  mutate(facet = "A: Effect of choice on evaluation of platform\namong those who did not choose to view less toxic political posts and\n those who likely would have chosen had they been offered the opportunity") %>% 
  mutate(model = factor(model, levels = c(  "likely_to_use_platform",
                                            "would_recommend_platform_to_friend",
                                            "satisfied_with_platform"),
                        labels = c(
                                "likely_to_use_platform" = "Using the platform",
                                "would_recommend_platform_to_friend" = "Recommending to friend",
                                "satisfied_with_platform" = "Satisfaction with\nConversationCircle"
                                )
                        ))

m3_a_plot <- ggplot(m3_a_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m3_a_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m3_a_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust=0)) +
  labs(x = "", y = "") 

m3_b_plot_data <- model_summary_for_plot(m3_b_1, m3_b_2, m3_b_3, m3_b_4, m3_b_5) %>% 
  mutate(facet = "B: Effect of choice on hostility ratings of content\namong those who did not choose to view less toxic political posts and\n those who likely would have chosen had they been offered the opportunity") %>% 
  mutate(model = factor(model, levels = c(  "out_party_avg_hostile",
                                            "in_party_avg_hostile",
                                            "nonpolitical_avg_hostile",
                                            "political_tweets_hostile_avg",
                                            "avg_hostility_score_all_posts"), 
                        labels=c(
    "out_party_avg_hostile" = "Out-party political posts",
    "in_party_avg_hostile" = "In-party political posts",
    "nonpolitical_avg_hostile" = "Non-partisan posts",
    "political_tweets_hostile_avg" = "Political posts",
    "avg_hostility_score_all_posts" = "All posts combined"
  ))) 

m3_b_plot <- ggplot(m3_b_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m3_b_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m3_b_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none",  axis.text.y = element_text(hjust=0)) +
  labs(x = "Estimate", y = "") 

m3_plot_data <- rbind(m3_a_plot_data, m3_b_plot_data)

ggplot(m3_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {m3_plot_data %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {m3_plot_data %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  labs(x = "Estimate", y = "") +
  facet_wrap(~facet, nrow = 2, scales = "free_y") +
    theme(legend.position = "none",  
        strip.text.x = element_text(face= "bold", size = 12)
        ) 
```

```{r, include = FALSE}
# ============================================================
# STUDY 1: EFFECT AMONG OPT-OUTS (Propensity Weighted)
# ============================================================

# Platform Evaluation (Study 1 Opt-outs)
stargazer::stargazer(m3_a_1, m3_a_2, m3_a_3,
  type = 'text',
  title = "Study 1: Effect of Choice Among Opt-Outs on Platform Evaluation",
  column.labels = c('Satisfaction', 'Likely to Use', 'Recommend'),
  dep.var.labels = 'Platform Evaluation (Propensity Weighted)',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"
)

# Hostility Ratings (Study 1 Opt-outs)
stargazer::stargazer(m3_b_1, m3_b_2, m3_b_3, m3_b_4, m3_b_5,
  type = 'text',
  title = "Study 1: Effect of Choice Among Opt-Outs on Hostility Ratings",
  column.labels = c('All Posts', 'Political', 'Non-political', 'In-party', 'Out-party'),
  dep.var.labels = 'Mean Hostility Rating (Propensity Weighted)',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Misinformation problem",
    "Banning users problem",
    "Shadow-banning problem",
    "Online bullying problem",
    "Tone/civility problem",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"
)

```


\clearpage

## Study 2

```{r s2-effects-of-choice-optouts-only, results = 'hold', echo = F, out.extra='angle=90', out.width = '100%', out.height='100%', fig.width = 9, fig.height = 4, fig.cap = "Study 2 Effects of choice on opt-outs."}
s2_m2_a_1_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "satisfied_with_platform"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT 
)}

s2_m2_a_2_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "likely_to_use_platform"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT
)}

s2_m2_a_3_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "would_recommend_platform_to_friend"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT
)}

s2_m2_b_1_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "avg_hostility_score_all_posts"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT
)}

s2_m2_b_2_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "political_tweets_hostile_avg"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT
)}

s2_m2_b_3_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "nonpolitical_avg_hostile"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT
)}
s2_m2_b_4_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "in_party_avg_hostile"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT
)}
s2_m2_b_5_optouts <- {lm(
  reformulate(rhs_reg_formula, 
              "out_party_avg_hostile"), 
  data = treatment_no_and_control2, 
  weights = algo_not_selected_prT
)}

s2_m2_a_plot_data_optouts <- model_summary_for_plot(s2_m2_a_1_optouts, s2_m2_a_2_optouts, s2_m2_a_3_optouts) %>% 
  mutate(facet = "A: Effect of choice on evaluation of platform in our follow-up study,\namong those who did opt-out and would have opted-out") %>% 
  mutate(model = factor(model, levels = c(  "likely_to_use_platform",
                                            "would_recommend_platform_to_friend",
                                            "satisfied_with_platform"),
                        labels = c(
                          "likely_to_use_platform" = "Using the platform",
                          "would_recommend_platform_to_friend" = "Recommending to friend",
                          "satisfied_with_platform" = "Satisfaction with\nConversationCircle"
                        )
  ))

s2_m2_a_plot_optouts <- ggplot(s2_m2_a_plot_data_optouts, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m2_a_plot_data_optouts %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m2_a_plot_data_optouts %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust=0)) +
  labs(x = "", y = "") 

s2_m2_b_plot_data_optouts <- model_summary_for_plot(s2_m2_b_1_optouts, s2_m2_b_2_optouts, s2_m2_b_3_optouts, s2_m2_b_4_optouts, s2_m2_b_5_optouts) %>% 
  mutate(facet = "B: Effect of choice on hostility ratings of content in our follow-up study,\namong those who did opt-out and would have opted-out") %>% 
  mutate(model = factor(model, levels = c(  "out_party_avg_hostile",
                                            "in_party_avg_hostile",
                                            "nonpolitical_avg_hostile",
                                            "political_tweets_hostile_avg",
                                            "avg_hostility_score_all_posts"), 
                        labels=c(
                          "out_party_avg_hostile" = "Out-party political posts",
                          "in_party_avg_hostile" = "In-party political posts",
                          "nonpolitical_avg_hostile" = "Non-partisan posts",
                          "political_tweets_hostile_avg" = "Political posts",
                          "avg_hostility_score_all_posts" = "All posts combined"
                        ))) 

s2_m2_b_plot_optouts <- ggplot(s2_m2_b_plot_data_optouts, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m2_b_plot_data_optouts %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m2_b_plot_data_optouts %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none",  axis.text.y = element_text(hjust=0)) +
  labs(x = "Estimate", y = "") 

s2_m2_plot_data_optouts <- rbind(s2_m2_a_plot_data_optouts, s2_m2_b_plot_data_optouts)

ggplot(s2_m2_plot_data_optouts, aes(x = value, y = model, color = model)) + 
  geom_line(data = {s2_m2_plot_data_optouts %>% filter(limit_type !="coefficient")}, mapping = aes( size = conf_level)) +
  geom_point(data = {s2_m2_plot_data_optouts %>% filter(limit_type=="coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  labs(x = "Estimate", y = "") +
  facet_wrap(~facet, nrow = 2, scales = "free_y") +
  theme(legend.position = "none",  
        strip.text.x = element_text(face= "bold", size = 12)
  ) 
```

\clearpage

# Comparing hostility ratings between two studies.

```{r}
combined_data <- rbind(
  {treatment_yes_and_control %>% 
      select(treatment_condition, 
             dem_and_leaning, 
             college, 
             strong_party_id2, 
             age,
             female, 
             white_only, 
             frequency_of_sm_use_numeric, 
             satisfied_with_sm, 
             misinfo_problem_numeric, 
             SM_banning_users_problem_numeric, 
             SM_shadow_banning_problem_numeric, 
             online_bullying_problem_numeric, 
             tone_civility_problem_numeric, 
             talk_to_others_abt_politics_frequently, 
             talk_to_others_abt_politics_infrequently, 
             satisfied_with_platform, 
             likely_to_use_platform, 
             would_recommend_platform_to_friend, 
             avg_hostility_score_all_posts, 
             political_tweets_hostile_avg, 
             nonpolitical_avg_hostile, 
             in_party_avg_hostile, 
             out_party_avg_hostile,
             positive_political_tweets_hostile_avg, 
             avg_hostility_score_positive_and_neutral_posts
             ) %>% 
      mutate(studyID = "a")},
  {treatment_yes_and_control2 %>% 
      select(treatment_condition, 
             dem_and_leaning, 
             college, 
             strong_party_id2, 
             age,
             female, 
             white_only, 
             frequency_of_sm_use_numeric, 
             satisfied_with_sm, 
             misinfo_problem_numeric, 
             SM_banning_users_problem_numeric, 
             SM_shadow_banning_problem_numeric, 
             online_bullying_problem_numeric, 
             tone_civility_problem_numeric, 
             talk_to_others_abt_politics_frequently, 
             talk_to_others_abt_politics_infrequently, 
             satisfied_with_platform, 
             likely_to_use_platform, 
             would_recommend_platform_to_friend, 
             avg_hostility_score_all_posts, 
             political_tweets_hostile_avg, 
             nonpolitical_avg_hostile, 
             in_party_avg_hostile, 
             out_party_avg_hostile, 
             positive_political_tweets_hostile_avg, 
             avg_hostility_score_positive_and_neutral_posts
             ) %>% 
      mutate(studyID = "b")})

combined_data <- combined_data %>% 
  mutate(studyID = as_factor(studyID)) 
#combined_data <- combined_data %>% mutate(studyID = relevel(studyID, ref ="b")) 


control_data_combined <- combined_data %>% 
  filter(treatment_condition == 0)

treated_yes_combined <- combined_data %>% 
  filter(treatment_condition == 1)


control_scores_combined <- predict(algo_selected_model, 
                                   newdata = control_data_combined, 
                                   type = "response")

# adding prop scores to control
control_data_combined <- control_data_combined %>% 
  mutate(
  algo_selected_att = control_scores_combined/(1-control_scores_combined),
  algo_selected_prT = control_scores_combined,
  algo_selected_thr = control_scores_combined>.5,
)

combined_data <- bind_rows(treated_yes_combined,control_data_combined) #GT edit
combined_data <- combined_data %>% 
    mutate(across(starts_with("algo_selected_"),function(x) ifelse(is.na(x),1,x)))

```


```{r}
rhs_reg_formula_combined_data <- "treatment_condition*studyID + dem_and_leaning + college + strong_party_id2 + age + female + white_only + frequency_of_sm_use_numeric + satisfied_with_sm + misinfo_problem_numeric + SM_banning_users_problem_numeric + SM_shadow_banning_problem_numeric + online_bullying_problem_numeric + tone_civility_problem_numeric + talk_to_others_abt_politics_frequently + talk_to_others_abt_politics_infrequently"
```

```{r}
# average score for positive posts in study 1
combined_data_positive_posts_hostility <- {lm(
    reformulate(rhs_reg_formula_combined_data, 
                "positive_political_tweets_hostile_avg"), 
    data = combined_data, 
weights = algo_selected_prT
  )}


combined_data_positive_and_neutral_posts_hostility <- {lm(
    reformulate(rhs_reg_formula_combined_data, 
                "avg_hostility_score_positive_and_neutral_posts"), 
    data = combined_data, 
weights = algo_selected_prT
  )}
```


```{r reg-table-m4-pt2, include = FALSE}
stargazer::stargazer(combined_data_positive_posts_hostility, combined_data_positive_and_neutral_posts_hostility, 
                     type = 'text',
                     title = "Regression table for combined dataset (Opt-ins from Study 1 and Study 2, along with their controls. Each regression is weighted by participants' probability of opting-in)",
                     column.labels = 
                       c('\\shortstack{Only positive political posts}',  
                         '\\shortstack{Positive and non-partisan posts}'),
                     dep.var.labels = 'Mean hostility ratings',
                     covariate.labels = c(
                       "Treatment condition",
                       "Study 2",
                       "Democrat",
                       "College educated",
                       "Strong party ID",
                       "Age",
                       "Female",
                       "White",
                       "Frequency of SM use",
                       "Satisfied with SM",
                     "Believe there is: \\\\  ~~misinformation problem\\\\~~on SM",
                       "\\\\ ~~a banning problem\\\\~~on SM",
                       "\\\\ ~~shadow-banning problem\\\\~~on SM",
                       "\\\\  ~~an onine bullying problem\\\\~~on SM",
                       "\\\\  ~~a civility problem\\\\~~on SM",
                       "Frequently talk to others\\\\about politics",
                       "Infrequently talk to others\\\\about politics",
                     "Treatment condition * Study 2"
                     ),
                     #single.row = TRUE,
                     column.sep.width = "1pt",
                     model.numbers = FALSE,
                     no.space = TRUE,
                     font.size = "scriptsize"
)
```

# Supplemental Analyses

This section reports additional analyses for affective polarization. It was included in our preregistration but is reported separately from the main manuscript due to space constraints.

\clearpage

## Affective Polarization

We measured affective polarization using feeling thermometers (0-100) toward Democratic and Republican voters. We computed in-party and out-party feeling thermometers based on respondents' partisanship, and calculated affective polarization as the difference between in-party and out-party warmth.

```{r affective-polarization-models}
# ----- STUDY 1: Overall Effect of Choice on Affective Polarization -----
ap_overall_s1 <- lm(reformulate(rhs_reg_formula, "affective_polarization"), 
                    data = data)
inparty_overall_s1 <- lm(reformulate(rhs_reg_formula, "inparty_ft"), 
                         data = data)
outparty_overall_s1 <- lm(reformulate(rhs_reg_formula, "outparty_ft"), 
                          data = data)

# ----- STUDY 1: Effect of Choice Among Opt-Ins -----
ap_optins_s1 <- lm(reformulate(rhs_reg_formula, "affective_polarization"), 
                   data = treatment_yes_and_control,
                   weights = algo_selected_prT)
inparty_optins_s1 <- lm(reformulate(rhs_reg_formula, "inparty_ft"), 
                        data = treatment_yes_and_control,
                        weights = algo_selected_prT)
outparty_optins_s1 <- lm(reformulate(rhs_reg_formula, "outparty_ft"), 
                         data = treatment_yes_and_control,
                         weights = algo_selected_prT)

# ----- STUDY 1: Effect of Choice Among Opt-Outs -----
ap_optouts_s1 <- lm(reformulate(rhs_reg_formula, "affective_polarization"), 
                   data = treatment_no_and_control,
                   weights = algo_not_selected_prT)
inparty_optouts_s1 <- lm(reformulate(rhs_reg_formula, "inparty_ft"), 
                        data = treatment_no_and_control,
                        weights = algo_not_selected_prT)
outparty_optouts_s1 <- lm(reformulate(rhs_reg_formula, "outparty_ft"), 
                         data = treatment_no_and_control,
                         weights = algo_not_selected_prT)

# ----- STUDY 2: Overall Effect of Choice on Affective Polarization -----
ap_overall_s2 <- lm(reformulate(rhs_reg_formula, "affective_polarization"), 
                    data = data2)
inparty_overall_s2 <- lm(reformulate(rhs_reg_formula, "inparty_ft"), 
                         data = data2)
outparty_overall_s2 <- lm(reformulate(rhs_reg_formula, "outparty_ft"), 
                          data = data2)

# ----- STUDY 2: Effect of Choice Among Opt-Ins -----
ap_optins_s2 <- lm(reformulate(rhs_reg_formula, "affective_polarization"), 
                   data = treatment_yes_and_control2,
                   weights = algo_selected_prT)
inparty_optins_s2 <- lm(reformulate(rhs_reg_formula, "inparty_ft"), 
                        data = treatment_yes_and_control2,
                        weights = algo_selected_prT)
outparty_optins_s2 <- lm(reformulate(rhs_reg_formula, "outparty_ft"), 
                         data = treatment_yes_and_control2,
                         weights = algo_selected_prT)

# ----- STUDY 2: Effect of Choice Among Opt-Outs -----
ap_optouts_s2 <- lm(reformulate(rhs_reg_formula, "affective_polarization"), 
                   data = treatment_no_and_control2,
                   weights = algo_not_selected_prT)
inparty_optouts_s2 <- lm(reformulate(rhs_reg_formula, "inparty_ft"), 
                        data = treatment_no_and_control2,
                        weights = algo_not_selected_prT)
outparty_optouts_s2 <- lm(reformulate(rhs_reg_formula, "outparty_ft"), 
                         data = treatment_no_and_control2,
                         weights = algo_not_selected_prT)
```

```{r ap-plot-s1-overall, fig.cap = "Study 1: Overall effect of choice on affective polarization", results = 'hold', out.width = '100%', fig.width = 8, fig.height = 3}
ap_s1_overall_plot_data <- model_summary_for_plot(ap_overall_s1, inparty_overall_s1, outparty_overall_s1) %>% 
  mutate(model = factor(model, 
                        levels = c("outparty_ft", "inparty_ft", "affective_polarization"),
                        labels = c("Out-party feeling\nthermometer", 
                                   "In-party feeling\nthermometer", 
                                   "Affective polarization\n(In-party - Out-party)")))

ggplot(ap_s1_overall_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {ap_s1_overall_plot_data %>% filter(limit_type != "coefficient")}, 
            mapping = aes(size = conf_level)) +
  geom_point(data = {ap_s1_overall_plot_data %>% filter(limit_type == "coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust = 0)) +
  labs(x = "Estimate", y = "", 
       title = "Study 1: Overall effect of choice on affective polarization")
```

```{r ap-plot-s1-optins, fig.cap = "Study 1: Effect of choice on affective polarization among opt-ins", results = 'hold', out.width = '100%', fig.width = 8, fig.height = 3}
ap_s1_optins_plot_data <- model_summary_for_plot(ap_optins_s1, inparty_optins_s1, outparty_optins_s1) %>% 
  mutate(model = factor(model, 
                        levels = c("outparty_ft", "inparty_ft", "affective_polarization"),
                        labels = c("Out-party feeling\nthermometer", 
                                   "In-party feeling\nthermometer", 
                                   "Affective polarization\n(In-party - Out-party)")))

ggplot(ap_s1_optins_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {ap_s1_optins_plot_data %>% filter(limit_type != "coefficient")}, 
            mapping = aes(size = conf_level)) +
  geom_point(data = {ap_s1_optins_plot_data %>% filter(limit_type == "coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust = 0)) +
  labs(x = "Estimate", y = "", 
       title = "Study 1: Effect of choice among opt-ins")
```

```{r ap-plot-s2-overall, fig.cap = "Study 2: Overall effect of choice on affective polarization", results = 'hold', out.width = '100%', fig.width = 8, fig.height = 3}
ap_s2_overall_plot_data <- model_summary_for_plot(ap_overall_s2, inparty_overall_s2, outparty_overall_s2) %>% 
  mutate(model = factor(model, 
                        levels = c("outparty_ft", "inparty_ft", "affective_polarization"),
                        labels = c("Out-party feeling\nthermometer", 
                                   "In-party feeling\nthermometer", 
                                   "Affective polarization\n(In-party - Out-party)")))

ggplot(ap_s2_overall_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {ap_s2_overall_plot_data %>% filter(limit_type != "coefficient")}, 
            mapping = aes(size = conf_level)) +
  geom_point(data = {ap_s2_overall_plot_data %>% filter(limit_type == "coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust = 0)) +
  labs(x = "Estimate", y = "", 
       title = "Study 2: Overall effect of choice on affective polarization")
```

```{r ap-plot-s2-optins, fig.cap = "Study 2: Effect of choice on affective polarization among opt-ins", results = 'hold', out.width = '100%', fig.width = 8, fig.height = 3}
ap_s2_optins_plot_data <- model_summary_for_plot(ap_optins_s2, inparty_optins_s2, outparty_optins_s2) %>% 
  mutate(model = factor(model, 
                        levels = c("outparty_ft", "inparty_ft", "affective_polarization"),
                        labels = c("Out-party feeling\nthermometer", 
                                   "In-party feeling\nthermometer", 
                                   "Affective polarization\n(In-party - Out-party)")))

ggplot(ap_s2_optins_plot_data, aes(x = value, y = model, color = model)) + 
  geom_line(data = {ap_s2_optins_plot_data %>% filter(limit_type != "coefficient")}, 
            mapping = aes(size = conf_level)) +
  geom_point(data = {ap_s2_optins_plot_data %>% filter(limit_type == "coefficient")}, size = 3) +
  geom_vline(xintercept = 0, linetype = "dashed", color = "black") +
  scale_size_manual(values = c("90" = 2, "95" = 1)) +
  theme_minimal() + 
  theme(legend.position = "none", axis.text.y = element_text(hjust = 0)) +
  labs(x = "Estimate", y = "", 
       title = "Study 2: Effect of choice among opt-ins")
```

```{r ap-regression-table, include = FALSE}
stargazer::stargazer(
  ap_overall_s1, ap_optins_s1, ap_overall_s2, ap_optins_s2,
  type = 'text',
  title = "Effect of Choice on Affective Polarization",
  column.labels = c('S1 Overall', 'S1 Opt-ins', 'S2 Overall', 'S2 Opt-ins'),
  dep.var.labels = 'Affective Polarization (In-party FT - Out-party FT)',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Believe misinformation problem on SM",
    "Believe banning problem on SM",
    "Believe shadow-banning problem on SM",
    "Believe online bullying problem on SM",
    "Believe civility problem on SM",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  column.sep.width = "1pt",
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"
)
```

```{r inparty-ft-regression-table, include = FALSE}
one <-inparty_overall_s1
two <-inparty_optins_s1
three <-inparty_overall_s2
four <- inparty_optins_s2
stargazer::stargazer(
 one, two, three, four,
  type = 'text',
  title = "Effect of Choice on In-Party Thermometer Ratings",
  column.labels = c('S1 Overall', 'S1 Opt-ins', 'S2 Overall', 'S2 Opt-ins'),
  #column.separate = c(1, 1, 1, 1),
  dep.var.labels = 'In-Party Feeling Thermometer Ratings',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Believe misinformation problem on SM",
    "Believe banning problem on SM",
    "Believe shadow-banning problem on SM",
    "Believe online bullying problem on SM",
    "Believe civility problem on SM",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  column.sep.width = "1pt",
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"
)
```

```{r outparty-ft-regression-table,  include = FALSE}
one <-outparty_overall_s1
two <-outparty_optins_s1
three <-outparty_overall_s2
four <- outparty_optins_s2
stargazer::stargazer(
 one, two, three, four,
  type = 'text',
  title = "Effect of Choice on Out-Party Thermometer Ratings",
  column.labels = c('S1 Overall', 'S1 Opt-ins', 'S2 Overall', 'S2 Opt-ins'),
  #column.separate = c(1, 1, 1, 1),
  dep.var.labels = 'Out-Party Feeling Thermometer Ratings',
  covariate.labels = c(
    "Treatment condition",
    "Democrat",
    "College educated",
    "Strong party ID",
    "Age",
    "Female",
    "White",
    "Frequency of SM use",
    "Satisfied with SM",
    "Believe misinformation problem on SM",
    "Believe banning problem on SM",
    "Believe shadow-banning problem on SM",
    "Believe online bullying problem on SM",
    "Believe civility problem on SM",
    "Frequently discuss politics",
    "Infrequently discuss politics"
  ),
  column.sep.width = "1pt",
  model.numbers = FALSE,
  no.space = TRUE,
  font.size = "scriptsize"
)
```


